﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;

namespace RTools_NTS.Util
{
	public class StreamTokenizer : IEnumerable<Token>, IEnumerable
	{
		public TextReader TextReader
		{
			get
			{
				return this.textReader;
			}
			set
			{
				this.textReader = value;
			}
		}

		public StreamTokenizerSettings Settings
		{
			get
			{
				return this.settings;
			}
		}

		public StreamTokenizer()
		{
			this.Initialize();
		}

		public StreamTokenizer(TextReader sr)
		{
			this.Initialize();
			this.textReader = sr;
		}

		public StreamTokenizer(string str)
		{
			this.Initialize();
			this.textReader = new StringReader(str);
		}

		private void Initialize()
		{
			this.backString = new CharBuffer(32);
			this.nextTokenSb = new CharBuffer(1024);
			this.InitializeStream();
			this.settings = new StreamTokenizerSettings();
			this.settings.SetDefaults();
			this.expSb = new CharBuffer();
			this.tmpSb = new CharBuffer();
		}

		private void InitializeStream()
		{
			this.lineNumber = 1;
			this.textReader = null;
		}

		public void Display()
		{
			this.Display(string.Empty);
		}

		public void Display(string prefix)
		{
			if (this.settings != null)
			{
				this.settings.Display(prefix + "    ");
			}
		}

		private StreamTokenizer.NextTokenState PickNextState(byte ctype, int c)
		{
			return this.PickNextState(ctype, c, StreamTokenizer.NextTokenState.Start);
		}

		private StreamTokenizer.NextTokenState PickNextState(byte ctype, int c, StreamTokenizer.NextTokenState excludeState)
		{
			if (c == 47)
			{
				return StreamTokenizer.NextTokenState.MaybeComment;
			}
			if (excludeState != StreamTokenizer.NextTokenState.MaybeHex && this.settings.ParseHexNumbers && c == 48)
			{
				return StreamTokenizer.NextTokenState.MaybeHex;
			}
			if (excludeState != StreamTokenizer.NextTokenState.MaybeNumber && this.settings.ParseNumbers && (this.settings.IsCharType(ctype, CharTypeBits.Digit) || c == 45 || c == 46))
			{
				return StreamTokenizer.NextTokenState.MaybeNumber;
			}
			if (this.settings.IsCharType(ctype, CharTypeBits.Word))
			{
				return StreamTokenizer.NextTokenState.Word;
			}
			if (this.settings.GrabEol && c == 10)
			{
				return StreamTokenizer.NextTokenState.Eol;
			}
			if (this.settings.IsCharType(ctype, CharTypeBits.Whitespace))
			{
				return StreamTokenizer.NextTokenState.Whitespace;
			}
			if (this.settings.IsCharType(ctype, CharTypeBits.Comment))
			{
				return StreamTokenizer.NextTokenState.LineComment;
			}
			if (this.settings.IsCharType(ctype, CharTypeBits.Quote))
			{
				return StreamTokenizer.NextTokenState.Quote;
			}
			if (c == StreamTokenizer.Eof || this.settings.IsCharType(ctype, CharTypeBits.Eof))
			{
				return StreamTokenizer.NextTokenState.Eof;
			}
			return StreamTokenizer.NextTokenState.Char;
		}

		private int GetNextChar()
		{
			int num;
			if (this.backString.Length > 0)
			{
				num = (int)this.backString[0];
				this.backString.Remove(0, 1);
				return num;
			}
			if (this.textReader == null)
			{
				return StreamTokenizer.Eof;
			}
			try
			{
				while ((num = this.textReader.Read()) == 13)
				{
				}
			}
			catch (Exception)
			{
				return StreamTokenizer.Eof;
			}
			if (num == 10)
			{
				this.lineNumber++;
			}
			else if (num < 0)
			{
				num = StreamTokenizer.Eof;
			}
			return num;
		}

		public bool NextToken(out Token token)
		{
			token = null;
			StreamTokenizer.NextTokenState nextTokenState = StreamTokenizer.NextTokenState.Start;
			int num = 0;
			if (this.nextTokenSb.Length > 0)
			{
				num = (int)this.nextTokenSb[this.nextTokenSb.Length - 1];
				byte ctype = this.settings.CharTypes[num];
				nextTokenState = this.PickNextState(ctype, num);
			}
			int num2 = 0;
			int num3 = 0;
			bool flag = false;
			int num4 = this.lineNumber;
			int num5 = num;
			bool flag2 = false;
			while (!flag2)
			{
				num = num5;
				num5 = this.GetNextChar();
				byte ctype2;
				if (num5 >= this.settings.CharTypes.Length)
				{
					ctype2 = 1;
				}
				else
				{
					ctype2 = this.settings.CharTypes[num5];
				}
				switch (nextTokenState)
				{
				case StreamTokenizer.NextTokenState.Start:
					nextTokenState = this.PickNextState(ctype2, num5);
					num4 = this.lineNumber;
					break;
				case StreamTokenizer.NextTokenState.Whitespace:
					if (!this.settings.IsCharType(ctype2, CharTypeBits.Whitespace) || (this.settings.GrabEol && num5 == 10))
					{
						if (this.settings.GrabWhitespace)
						{
							token = new WhitespaceToken(this.nextTokenSb.ToString(), num4);
							flag2 = true;
							this.nextTokenSb.Length = 0;
						}
						else
						{
							this.nextTokenSb.Length = 0;
							num4 = this.lineNumber;
							nextTokenState = this.PickNextState(ctype2, num5);
						}
					}
					break;
				case StreamTokenizer.NextTokenState.Word:
					if (!this.settings.IsCharType(ctype2, CharTypeBits.Word) && !this.settings.IsCharType(ctype2, CharTypeBits.Digit))
					{
						token = new WordToken(this.nextTokenSb.ToString(), num4);
						flag2 = true;
						this.nextTokenSb.Length = 0;
					}
					break;
				case StreamTokenizer.NextTokenState.Quote:
					if (num5 == (int)this.nextTokenSb[0])
					{
						int num6 = 0;
						int num7 = this.nextTokenSb.Length - 1;
						while (num7 >= 0 && this.nextTokenSb[num7] == '\\')
						{
							num6++;
							num7--;
						}
						if (num6 % 2 == 0)
						{
							nextTokenState = StreamTokenizer.NextTokenState.EndQuote;
						}
					}
					if (nextTokenState != StreamTokenizer.NextTokenState.EndQuote && num5 == StreamTokenizer.Eof)
					{
						if (this.settings.DoUntermCheck)
						{
							this.nextTokenSb.Length = 0;
							throw new StreamTokenizerUntermQuoteException("Unterminated quote");
						}
						token = new QuoteToken(this.nextTokenSb.ToString(), num4);
						flag2 = true;
						this.nextTokenSb.Length = 0;
					}
					break;
				case StreamTokenizer.NextTokenState.EndQuote:
					token = new QuoteToken(this.nextTokenSb.ToString(), num4);
					flag2 = true;
					this.nextTokenSb.Length = 0;
					break;
				case StreamTokenizer.NextTokenState.MaybeNumber:
				{
					bool flag3 = false;
					if (this.settings.IsCharType(ctype2, CharTypeBits.Digit) || this.settings.IsCharType(num, CharTypeBits.Digit))
					{
						flag = true;
					}
					if (num5 == 46)
					{
						num2++;
						if (num2 > 1)
						{
							flag3 = true;
						}
					}
					else if (num5 == 101 || num5 == 69)
					{
						num3++;
						if (!flag)
						{
							flag3 = true;
						}
						else if (num3 > 1)
						{
							flag3 = true;
						}
						else
						{
							flag3 = true;
							this.expSb.Clear();
							this.expSb.Append((char)num5);
							char c;
							if (this.GrabInt(this.expSb, true, out c))
							{
								this.nextTokenSb.Append(this.expSb);
								num5 = (int)c;
							}
						}
					}
					else if (num5 == StreamTokenizer.Eof)
					{
						flag3 = true;
					}
					else if ((!this.settings.IsCharType(ctype2, CharTypeBits.Digit) && num5 != 101 && num5 != 69 && num5 != 45 && num5 != 46) || (num5 == 43 && num3 == 0))
					{
						flag3 = true;
					}
					else if (num5 == 45 && num != 101 && num != 69)
					{
						flag3 = true;
					}
					if (flag3)
					{
						if (flag)
						{
							if (this.nextTokenSb.IndexOf('.') >= 0 || this.nextTokenSb.IndexOf('e') >= 0 || this.nextTokenSb.IndexOf('E') >= 0 || this.nextTokenSb.Length >= 19)
							{
								token = new FloatToken(this.nextTokenSb.ToString(), num4);
							}
							else
							{
								token = new IntToken(this.nextTokenSb.ToString(), num4);
							}
							flag2 = true;
							this.nextTokenSb.Length = 0;
						}
						else
						{
							this.nextTokenSb.Append((char)num5);
							this.backString.Append(this.nextTokenSb);
							this.nextTokenSb.Length = 0;
							num5 = (int)this.backString[0];
							this.backString.Remove(0, 1);
							nextTokenState = this.PickNextState(this.settings.CharTypes[num5], num5, StreamTokenizer.NextTokenState.MaybeNumber);
						}
					}
					break;
				}
				case StreamTokenizer.NextTokenState.MaybeComment:
					if (num5 == StreamTokenizer.Eof)
					{
						token = new CharToken(this.nextTokenSb.ToString(), num4);
						flag2 = true;
						this.nextTokenSb.Length = 0;
					}
					else if (this.settings.SlashSlashComments && num5 == 47)
					{
						nextTokenState = StreamTokenizer.NextTokenState.LineComment;
					}
					else if (this.settings.SlashStarComments && num5 == 42)
					{
						nextTokenState = StreamTokenizer.NextTokenState.BlockComment;
					}
					else
					{
						token = new CharToken(this.nextTokenSb.ToString(), num4);
						flag2 = true;
						this.nextTokenSb.Length = 0;
					}
					break;
				case StreamTokenizer.NextTokenState.MaybeHex:
					if (num5 != 120)
					{
						this.nextTokenSb.Append((char)num5);
						this.backString.Append(this.nextTokenSb);
						this.nextTokenSb.Length = 0;
						num5 = (int)this.backString[0];
						this.backString.Remove(0, 1);
						nextTokenState = this.PickNextState(this.settings.CharTypes[num5], num5, StreamTokenizer.NextTokenState.MaybeHex);
					}
					else
					{
						nextTokenState = StreamTokenizer.NextTokenState.HexGot0x;
					}
					break;
				case StreamTokenizer.NextTokenState.HexGot0x:
					if (!this.settings.IsCharType(ctype2, CharTypeBits.HexDigit))
					{
						this.nextTokenSb.Append((char)num5);
						this.backString.Append(this.nextTokenSb);
						this.nextTokenSb.Length = 0;
						num5 = (int)this.backString[0];
						this.backString.Remove(0, 1);
						nextTokenState = this.PickNextState(this.settings.CharTypes[num5], num5, StreamTokenizer.NextTokenState.MaybeHex);
					}
					else
					{
						nextTokenState = StreamTokenizer.NextTokenState.HexNumber;
					}
					break;
				case StreamTokenizer.NextTokenState.HexNumber:
					if (!this.settings.IsCharType(ctype2, CharTypeBits.HexDigit))
					{
						token = IntToken.ParseHex(this.nextTokenSb.ToString(), num4);
						flag2 = true;
						this.nextTokenSb.Length = 0;
					}
					break;
				case StreamTokenizer.NextTokenState.LineComment:
					if (num5 == StreamTokenizer.Eof)
					{
						if (this.settings.GrabComments)
						{
							token = new CommentToken(this.nextTokenSb.ToString(), num4);
							flag2 = true;
							this.nextTokenSb.Length = 0;
						}
						else
						{
							this.nextTokenSb.Length = 0;
							num4 = this.lineNumber;
							nextTokenState = this.PickNextState(ctype2, num5);
						}
					}
					else if (num5 == 10)
					{
						if (this.settings.GrabComments)
						{
							token = new CommentToken(this.nextTokenSb.ToString(), num4);
							flag2 = true;
							this.nextTokenSb.Length = 0;
						}
						else
						{
							this.nextTokenSb.Length = 0;
							num4 = this.lineNumber;
							nextTokenState = this.PickNextState(ctype2, num5);
						}
					}
					break;
				case StreamTokenizer.NextTokenState.BlockComment:
					if (num5 == StreamTokenizer.Eof)
					{
						if (this.settings.DoUntermCheck)
						{
							this.nextTokenSb.Length = 0;
							throw new StreamTokenizerUntermCommentException("Unterminated comment.");
						}
						if (this.settings.GrabComments)
						{
							token = new CommentToken(this.nextTokenSb.ToString(), num4);
							flag2 = true;
							this.nextTokenSb.Length = 0;
						}
						else
						{
							this.nextTokenSb.Length = 0;
							num4 = this.lineNumber;
							nextTokenState = this.PickNextState(ctype2, num5);
						}
					}
					else if (num5 == 47 && num == 42)
					{
						nextTokenState = StreamTokenizer.NextTokenState.EndBlockComment;
					}
					break;
				case StreamTokenizer.NextTokenState.EndBlockComment:
					if (this.settings.GrabComments)
					{
						token = new CommentToken(this.nextTokenSb.ToString(), num4);
						flag2 = true;
						this.nextTokenSb.Length = 0;
					}
					else
					{
						this.nextTokenSb.Length = 0;
						num4 = this.lineNumber;
						nextTokenState = this.PickNextState(ctype2, num5);
					}
					break;
				case StreamTokenizer.NextTokenState.Char:
					token = new CharToken((char)num, num4);
					flag2 = true;
					this.nextTokenSb.Length = 0;
					break;
				case StreamTokenizer.NextTokenState.Eol:
					token = new EolToken(num4 - 1);
					flag2 = true;
					this.nextTokenSb.Length = 0;
					break;
				case StreamTokenizer.NextTokenState.Eof:
					token = new EofToken(num4);
					this.nextTokenSb.Length = 0;
					return false;
				case StreamTokenizer.NextTokenState.Invalid:
					return false;
				default:
					return false;
				}
				if (num5 != StreamTokenizer.Eof)
				{
					this.nextTokenSb.Append((char)num5);
					continue;
				}
			}
			return true;
		}

		private bool GrabInt(CharBuffer sb, bool allowPlus, out char thisChar)
		{
			this.tmpSb.Clear();
			thisChar = (char)this.GetNextChar();
			if ((int)thisChar == StreamTokenizer.Eof)
			{
				return false;
			}
			if (thisChar == '+')
			{
				if (!allowPlus)
				{
					this.backString.Append(thisChar);
					return false;
				}
				this.tmpSb.Append(thisChar);
			}
			else if (thisChar == '-')
			{
				this.tmpSb.Append(thisChar);
			}
			else
			{
				if (!this.settings.IsCharType(thisChar, CharTypeBits.Digit))
				{
					this.backString.Append(thisChar);
					return false;
				}
				this.backString.Append(thisChar);
			}
			bool flag = false;
			while ((int)(thisChar = (char)this.GetNextChar()) != StreamTokenizer.Eof && this.settings.IsCharType(thisChar, CharTypeBits.Digit))
			{
				flag = true;
				this.tmpSb.Append(thisChar);
			}
			if (flag)
			{
				sb.Append(this.tmpSb);
				return true;
			}
			this.backString.Append(this.tmpSb);
			if ((int)thisChar != StreamTokenizer.Eof)
			{
				this.backString.Append(thisChar);
			}
			return false;
		}

		public bool Tokenize(IList<Token> tokens)
		{
			this.lineNumber = 1;
			Token token;
			while (this.NextToken(out token))
			{
				if (token == null)
				{
					throw new NullReferenceException("StreamTokenizer: Tokenize: Got a null token from NextToken.");
				}
				tokens.Add(token);
			}
			tokens.Add(token);
			return true;
		}

		public bool TokenizeReader(TextReader tr, IList<Token> tokens)
		{
			this.textReader = tr;
			return this.Tokenize(tokens);
		}

		public bool TokenizeFile(string fileName, IList<Token> tokens)
		{
			FileInfo fileInfo = new FileInfo(fileName);
			FileStream fileStream = null;
			try
			{
				fileStream = fileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.None);
				this.textReader = new StreamReader(fileStream);
			}
			catch (DirectoryNotFoundException)
			{
			}
			try
			{
				if (!this.Tokenize(tokens))
				{
					this.textReader.Close();
					if (fileStream != null)
					{
						fileStream.Close();
					}
					return false;
				}
			}
			catch (StreamTokenizerUntermException ex)
			{
				this.textReader.Close();
				if (fileStream != null)
				{
					fileStream.Close();
				}
				throw ex;
			}
			if (this.textReader != null)
			{
				this.textReader.Close();
			}
			if (fileStream != null)
			{
				fileStream.Close();
			}
			return true;
		}

		public Token[] TokenizeFile(string fileName)
		{
			List<Token> list = new List<Token>();
			if (!this.TokenizeFile(fileName, list))
			{
				return null;
			}
			if (list.Count > 0)
			{
				return list.ToArray();
			}
			return null;
		}

		public bool TokenizeString(string str, IList<Token> tokens)
		{
			this.textReader = new StringReader(str);
			return this.Tokenize(tokens);
		}

		public bool TokenizeStream(Stream s, IList<Token> tokens)
		{
			this.textReader = new StreamReader(s);
			return this.Tokenize(tokens);
		}

		public IEnumerator<Token> GetEnumerator()
		{
			this.lineNumber = 1;
			Token token;
			while (this.NextToken(out token))
			{
				if (token == null)
				{
					throw new NullReferenceException("StreamTokenizer: Tokenize: Got a null token from NextToken.");
				}
				yield return token;
			}
			yield return token;
			yield break;
		}

		IEnumerator IEnumerable.GetEnumerator()
		{
			return this.GetEnumerator();
		}

		public static readonly int NChars = 128;

		private static readonly int Eof = StreamTokenizer.NChars;

		private TextReader textReader;

		private int lineNumber;

		private CharBuffer backString;

		private CharBuffer nextTokenSb;

		private CharBuffer tmpSb;

		private CharBuffer expSb;

		private StreamTokenizerSettings settings;

		private enum NextTokenState
		{
			Start,
			Whitespace,
			Word,
			Quote,
			EndQuote,
			MaybeNumber,
			MaybeComment,
			MaybeHex,
			HexGot0x,
			HexNumber,
			LineComment,
			BlockComment,
			EndBlockComment,
			Char,
			Eol,
			Eof,
			Invalid
		}
	}
}
